import asyncio
import functools
import time

from aiohttp import ClientSession
from lxml import etree
from pymongo import MongoClient


class AsyncQFangSpider:
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',
            'Cookie': 'sid=1130e471-e3b4-47a0-80db-e2d09486ca3c; qchatid=3067c6db-a210-4439-b704-293e23ecadcb; cookieId=b9beb8f3-f860-46d1-aa91-674eda0e3979; JSESSIONID=aaaTxz65dcXGa40ev43wx; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1605143476; _ga=GA1.3.1464733272.1605143476; _gid=GA1.3.141663440.1605143476; WINDOW_DEVICE_PIXEL_RATIO=1.25; CITY_NAME=NANJING; Hm_lvt_eaaf2b6886b56980102bdc75a8151d46=1605143478; _jzqc=1; _jzqckmp=1; _qzjc=1; _jzqa=1.1213954754549273900.1605143492.1605162677.1605164649.6; _jzqx=1.1605164649.1605164649.1.jzqsr=nanjing%2Eqfang%2Ecom|jzqct=/sale/f1.-; wzws_cid=7022d59b032407a01e6f757e88efc2a628b18568f64e9f939200f3d0f06a3b027dee900f749de020346fdedfdb5644e00bcb78509ad71ab28a9f458d4c62ea5bf8331defdeeab3941d4c98a2451a1ef0; Hm_lpvt_eaaf2b6886b56980102bdc75a8151d46=1605165084; _dc_gtm_UA-47416713-1=1; _qzja=1.1708129336.1605143491636.1605162677032.1605164649066.1605164992089.1605165083866.0.0.0.18.6; _qzjb=1.1605164649065.8.0.0.0; _qzjto=18.6.0; _jzqb=1.8.10.1605164649.1; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1605165084; SALEROOMREADRECORDCOOKIE=100272634; looks=SALE%2C100272634%2C2950'
        }
        self.client = MongoClient('localhost', 27017)

    async def q_fang_spider(self, url):
        async with ClientSession() as session:
            async with session.get(url=url, headers=self.headers) as r:
                return await r.text()

    def processing_data(self, task_obj, page):
        resp = task_obj.result()
        collection_name = 'q_fang_info_page_' + page
        collection = self.client['q_fang'][collection_name]
        html = etree.HTML(resp)
        house_li_list = html.xpath('//div[@class="list-result"]/ul/li')
        house_info_list = []
        for li in house_li_list:
            new_dict = {}
            title = li.xpath('./div[2]/div[1]/a/@title')[0]
            title = title.replace('\xa0', '')
            new_dict['title'] = title

            house_type = li.xpath('.//div[@class="house-metas clearfix"]/p[1]/text()')[0]
            new_dict['house_type'] = house_type

            size = li.xpath('.//div[@class="house-metas clearfix"]/p[2]/text()')[0]
            new_dict['size'] = size

            floor = li.xpath('.//div[@class="house-metas clearfix"]/p[4]/text()')[0]
            floor = floor.replace('\r', '').replace('\n', '').replace('\t', '')
            new_dict['floor'] = floor

            orientation = li.xpath('.//div[@class="house-metas clearfix"]/p[5]/text()')[0]
            new_dict['orientation'] = orientation

            create_time = li.xpath('.//div[@class="house-metas clearfix"]/p[6]/text()')
            if create_time:
                create_time = create_time[0].replace('\r', '').replace('\n', '').replace(' ', '')
                new_dict['create_time'] = create_time
            else:
                new_dict['create_time'] = '该房源还没有上传建立时间'

            total_price = li.xpath('.//div[@class="list-price"]/p[1]/span[1]/text()')[0]
            new_dict['total_price'] = total_price + '万'

            unit_price = li.xpath('.//div[@class="list-price"]/p[2]/text()')[0]
            new_dict['unit_price'] = unit_price

            qu = li.xpath('.//div[@class="text fl"]/a[1]/text()')[0]
            lu = li.xpath('.//div[@class="text fl"]/a[2]/text()')[0]
            hao = li.xpath('.//div[@class="text fl"]/a[3]/text()')[0]
            addr = qu + '-' + lu + '-' + hao
            new_dict['addr'] = addr

            school = li.xpath('.//div[@class="house-tags clearfix"]/a/span/text()')
            if school:
                new_dict['school'] = school[0]
            else:
                new_dict['school'] = '该房源附近没有学校'

            station_house = li.xpath('.//div[@class="house-tags clearfix"]/p/text()')
            str1 = ''
            for tag in station_house:
                str1 += tag + ','
            str1 = str1[:-1]
            new_dict['tag'] = str1

            house_info_list.append(new_dict)

        # print(house_info_list)

        collection.insert_many(house_info_list)
        print('第%s页爬取完成' % page)

    async def main(self):
        tasks = []
        for i in range(1, 11):
            url = 'https://nanjing.qfang.com/sale/f' + str(i)
            task_q_fang = asyncio.create_task(self.q_fang_spider(url))
            task_q_fang.add_done_callback(functools.partial(self.processing_data, page=str(i)))

            tasks.append(task_q_fang)
        await asyncio.wait(tasks)


if __name__ == '__main__':
    start = time.time()
    async_q_fang_spider = AsyncQFangSpider()
    asyncio.run(async_q_fang_spider.main())
    over = time.time()
    jie = over - start
    print(jie)
